#!C:\Python26

from pluginManage import Plugin


import BeautifulSoup
import sys
import time
import re
import urlparse
import os.path
import cPickle
import bs4

import urllib

import logmanager

import random
random.seed()

import copy

import Queue
import threading



class getFA(Plugin):

	capabilities			=	"wScraper"									# Used for detecting ifany random .py file is a plugin file
	version				=	1.0										# Used for detecting available configuration options
	pluginName			=	"FAscrape"									# Plugin Name variable
	siteName			=	"FurAffinity"								# Value for populating "Art Site" dropbox
	nameSource			=	["Single Name", "Name List File"]						# namesource value passed to plugin
	nameSourceEN			=	1										# Determines if Name Source input box is enabled (Boolean, 0-1)
	artistNameEN			=	1										# Determines if Artist Name input box is enabled (Boolean, 0-1)
	artistListEN			=	1										# Determines if Artist List input box is enabled (Boolean, 0-1)
	overwriteModes			=	["Check Folders", "Check Files w/Memory and Time",
						"Check Files w/Memory", "Check Files", "Overwrite Files"]			# Determines if overwrite mode dropbox is enabled for plugin
	retrievalHandles		=	2										# How may http retrieval handles app wants

	DLFolder			=	"Fur Affinity"
	logFileName			=	"LogFile.txt"
	pickleFileName			=	"Pickle.dat"

	cookieCapable = 1

	#log = logmanager.ThreadLog(retrievalHandles)

	waitTimer = 2.5

	def checkCookie(self, cookiejar):
		userID = re.search("<Cookie a=[0-9a-f\-]*? for \.furaffinity\.net/>", "%s" % cookiejar)
		sessionID = re.search("<Cookie b=[0-9a-f\-]*? for \.furaffinity\.net/>", "%s" % cookiejar)
		if userID and sessionID:
			return "Have FA Cookies:\n	%s\n	%s" % (userID.group(0), sessionID.group(0))

		return "Do not have FA login Cookies"

	def getCookie(self, userName, password, opener):


		logondict = {"retard_protection" : "1", "action" : "login", "name" : userName, "pass" : password}

		#print logondict

		params = urllib.urlencode(logondict)
		page = opener.open('https://www.furaffinity.net/login/?url=/', params)
		pagetext = page.read()
		page.close()

		if re.search("You have typed in an erroneous username or password, please try again", pagetext):
			return "Login Failed"
		else:
			return "Logged In"



	def getGalleryLst(self, url, getHandle):
		flag = 1


		artlinks = []								# Declare array of links
		self.config.progBars.pulseSubBar()

		while flag:
			turl = "%s%s%s" % (url, flag, "/")
			self.log.info("Getting = " + turl)
			mpgctnt = getHandle.getpage(turl)							# Request Image
			if mpgctnt == "Failed":
					self.log.error("Cannot get Page: %s" % turl)
					return []

			self.config.progBars.pulseSubBar()
			#print mpgctnt
			mpgctnt = mpgctnt.replace("\\", "")					# Scrub out \/ negations from links in javascript
			regx = re.compile("<script.+?>")					# Scrub Out <Script> Tags, which break Beautiful Soup
			mpgctnt = regx.sub("" , mpgctnt)

			self.config.progBars.pulseSubBar()


			BeautifulSoup.soup = BeautifulSoup.BeautifulSoup(mpgctnt)

			self.config.progBars.pulseSubBar()

			anchortags = BeautifulSoup.soup("a")

			self.config.progBars.pulseSubBar()

			temp = ""								# Temporary value for removing duplicate links
			artcnt = 0								# track number of art pages (for determing gallery size)


			for tag in anchortags:
				#print tag
				try:
					nudderlink = urlparse.urljoin(url, tag['href'])			# Extract link
				except KeyError:							# badly formed link ? probably just a named anchor like '<a name="something">'
					continue
				if (nudderlink.find("/view/") + 1) and nudderlink != temp:		# See if link is to art page, and if it's a duplicate
					temp = nudderlink
					artcnt = artcnt + 1
					artlinks.append(nudderlink)

			self.config.progBars.pulseSubBar()

			flag = flag + 1
			if artcnt == 0:
				flag = 0

			time.sleep(self.waitTimer)

			self.config.progBars.pulseSubBar()

		self.log.info("Found %s links" % (len(artlinks)))


		return artlinks



	def getpic(self, link, getHandle, faName):



		imgurl = ""

		mpgctnt = getHandle.getpage(link)					# Get Webpage
		#soup = bs4.BeautifulSoup(mpgctnt, "lxml")

		if mpgctnt == "Failed":
			self.log.error("cannot get page: %s" % link)
			return "Failed"



		regx1 = re.compile('[\"\'](https://d\.facdn\.net\/[^\'\"]*?)[\"\']')
		reResult = regx1.search(mpgctnt)

		if reResult:
			imgurl = reResult.group(1)
			self.log.info("Found direct download URL : %s" % imgurl)

		if imgurl == "" :

			regx2 = re.compile('var full_url *?= [\'\"]([^\'\"]*?)[\"\']')							# Extract Image location from javascript
			reResult = regx2.search(mpgctnt)


			if reResult:
				imgurl = reResult.group(1)
				self.log.info("Found Image URL : %s" % imgurl)

		if imgurl == "" :
			regx2 = re.compile('<param name="movie" *?value=[\'\"]([^\s\'\"]*?)[\"\']')

			reResult = regx2.search(mpgctnt)
			if reResult:
				imgurl = reResult.group(1)

				self.log.info("Found Flash URL : %s" % imgurl)

		#downloadLink = soup.find("td", class_="alt1")
		#print downloadLink


		if imgurl == "" :
			self.log.error("OH NOES!!! No image on page: %s" % link)

			return "Failed"								# Return Fail
		else:

			if not "http:" in imgurl:						# Fa is for some bisarre reason, leaving the 'http:' off some of their URLs
				imgurl = "http:%s" % imgurl

			fileTypeRe = re.compile(".+\.")
			fileNameRe = re.compile(".+/")						# Pull out filename only

			fname = fileNameRe.sub("" , imgurl)
			ftype = fileTypeRe.sub("" , fname)					# Pull out filename only

			self.log.info("			Filename = " + fname)
			self.log.info("			File Type = " + ftype)
			self.log.info("			FileURL  = " + imgurl)

			try:
				DLFolderDecoded = unicode(self.DLFolder)
				faNameDecoded = unicode(faName)
				fnameDecoded = unicode(fname)
			except:

				print "Text encoding issue"
				print faName

				import traceback
				traceback.print_exc()

				self.log.error("Text encoding issue")
				self.log.error("%s" % faName)
				self.log.error("%s" % link)
				self.log.error("Error with text decoding")
				print "Skipping Errored page"
				return "Failed"

			try:
				filePath = os.path.join(DLFolderDecoded, faNameDecoded, fnameDecoded)
			except:
				print "file path issue"

				import traceback
				traceback.print_exc()

				self.log.error("file path issue")
				self.log.error("%s" % faName)
				self.log.error("%s" % link)
				self.log.exception("Error with path joining")
				return "Failed"

			if (not os.access(filePath, os.F_OK)) or self.config.ovwMode == "Overwrite Files" or os.path.getsize(filePath) == 0:		# short-circuit evaluation means that if the file
																		# does not exist, the if evaluates to true, so
																		# os.path.getsize never gets called, and we
																		# don't get an error

				try:


					if not os.access(filePath, os.W_OK):
						open(filePath, 'wb').close()
						os.unlink(filePath)
				except:
					print "Wut? String Printing Issue..."
					self.log.exception("Wut? String Printing Issue...")
					try:
						fname = urllib.quote_plus(fname)
						print "Strings = \'%s\' - \'%s\' - \'%s\' " % (self.DLFolder, faName, fname)
						print "File Path = \'%s\'" % filePath
						self.log.exception("Strings = \'%s\' - \'%s\' - \'%s\' ", self.DLFolder, faName, fname)

						filePath = os.path.join(self.DLFolder, faName, fname)

						if not os.access(filePath, os.W_OK):
							open(filePath, 'wb').close()
							os.unlink(filePath)



					except:
						print "error error"
						self.log.exception("error error")
						return "Failed"




				imgdat = getHandle.getpage(imgurl)							# Request Image

				if imgdat == "Failed":
					self.log.error("cannot get image %s" % imgurl)
					self.log.error("source gallery page: %s" % link)
					return "Failed"

				errs = 0
				imgFile = None

				while not imgFile:
					try:
						imgFile = open(filePath, "wb")						# Open file for saving image (Binary)
						imgFile.write(imgdat)							# Write Image to File
						imgFile.close()
					except IOError:
						try:
							imgFile.close()
						except:
							pass
						errs += 1
						self.log.critical("Error attempting to save image file - %s" % filePath)
						if errs > 3:
							self.log.critical("Could not open file for writing!")
							return "Failed"



				self.log.info("Successfully got: " + imgurl)
				return "Downloaded"									# Return Success

			else:
				self.log.info("Exists, skipping...")
				return "Exists"



	def getFA(self, faName, getHandle):									# Main Function Call



		url = "http://www.furaffinity.net/gallery/" + faName + "/"						# Setup for FA Page
		scrapsUrl = "http://www.furaffinity.net/scraps/" + faName + "/"						# Setup for FA "Scraps" Page



		if os.access(os.path.join(self.DLFolder, faName), os.W_OK) and self.config.ovwMode == "Check Folders":
			self.log.info("Folder " + faName + "/ Exists: ")
			self.log.info("Assuming page has already been downloaded, skipping")

		else:
			if not os.access(os.path.join(self.DLFolder, faName), os.W_OK):
				try:
					os.mkdir(os.path.join(self.DLFolder, faName))
					self.log.info("Created folder " + faName)
					self.log.info("Beginning Download")
				except:
					self.log.error("Cannot Make working directory %s/. Do you have write Permissions? %s" % (faName, sys.exc_info()[0]))
					return [], []

			self.log.info("Base Gallery URL " + url)							# Show Website that's being scrubbed




			if self.config.ovwMode == "Check Files w/Memory and Time" or self.config.ovwMode == "Check Files w/Memory":
				try:
					pikPath = os.path.join(self.DLFolder, faName, self.pickleFileName)
					print pikPath

					pickleFile = open(pikPath, "rb")
					pickleFileCtnt = cPickle.load(pickleFile)
					pickleFile.close()

					previousScanTime = 0
					pickleFileLinks = []

					if type(pickleFileCtnt) == list:
						pickleFileLinks = pickleFileCtnt
						print "Picklefile is old format. Converting"

					elif type(pickleFileCtnt) == dict:
						pickleFileLinks = pickleFileCtnt["links"]
						previousScanTime = pickleFileCtnt["date"]

					else:
						print "Pickle file corrupted?"
						print pickleFileCtnt

					if self.config.ovwMode == "Check Files w/Memory and Time":
						if (time.time() - previousScanTime) < (60 * 60 * 24 * 3):
							self.log.info("Artist %s has been scanned within the last seven days - Skipping" % faName)
							return [], []





				except:
					self.log.info("No Pickle file - Is this the first run on this username?")
					pickleFileLinks = []
			else:
				pickleFileLinks = []

							# 60 seconds in a minute * 60 minutes in an hour * 24 hours in a day * 3 days


			artLinksAll = []
			artLinksAll.extend(self.getGalleryLst(url, getHandle))
			artLinksAll.extend(self.getGalleryLst(scrapsUrl, getHandle))

			self.log.info("Found %s Links in total." % len(artLinksAll))

			if artLinksAll == []:

				self.log.error("Artist %s has no links!" % faName)
				return [], []

			newLinks = []
			for link in artLinksAll:
				if link in pickleFileLinks:
					#self.log.info( "Already Have: "+link)
					pass
				else:
					newLinks.append(link)


			self.log.info("Found %s new pieces of content " % len(newLinks))


			#pickleFile = open(os.path.join(self.DLFolder, faName, self.pickleFileName), "wb")		#Nuke pickle file so if the software crashes, it'll try to get everything the next time
			#cPickle.dump("", pickleFile)
			#pickleFile.close()


			self.config.progBars.setSubBarLen(len(newLinks))


			successlinks = []
			existlinks = []
			faillinks = []


			for link in newLinks:
				success = self.getpic(link, getHandle, faName)					# Scrape Pictures!
				self.config.progBars.incSubBar()

				if success == "Downloaded":
					successlinks.append(link)
				elif success == "Exists":
					existlinks.append(link)
				else:
					faillinks.append(link)

				time.sleep(self.waitTimer)

			if faillinks:
				self.log.info("- - - Failed Pages: %s - - -" % (len(faillinks)))
				for link in faillinks:
					if link:
						self.log.info("Failed - %s" % link)


			try:
				validLinks = copy.copy(successlinks)
				validLinks += existlinks
				validLinks += pickleFileLinks

			except:
				validLinks = []
				print "WTF"
				import traceback
				traceback.print_exc()

				print "successlinks"
				print successlinks
				print "existlinks"
				print existlinks
				print "pickleFileLinks"
				print pickleFileLinks

			#print validLinks

			pickleFile = open(os.path.join(self.DLFolder, faName, self.pickleFileName), "wb")

			dumpFileContent = {"date" : time.time(), "links" : validLinks}

			cPickle.dump(dumpFileContent, pickleFile)
			pickleFile.close()

			if faillinks:
				print faName, faillinks
			return faillinks, successlinks

	def retFalse():
		return False

	def threadSleep(self, sleepTime, maxTime, breakCheck=retFalse):


		self.config.progBars.addSubProgressBar()
		self.config.progBars.setSubBarLen(maxTime, value=sleepTime)

		self.log.info("Thread is sleeping %s seconds" % sleepTime)

		while not self.config.stopChildThreads and not breakCheck() and sleepTime:				# we don't want to sleep for many minutes if the program is trying to exit, or the name queue is empty
			time.sleep(1)											# Sleep somewhere between 1 and 10 minutes
			sleepTime -= 1
			infoString = "Sleeping %s seconds" % sleepTime
			self.config.progBars.setSubBarLen(maxTime, value=sleepTime, infoString=infoString)



		self.config.progBars.removeSubProgressBar()


	def childThread(self, getHandle):
		self.log.info("Thread HTTP user agent - %s" % (getHandle.browserHeaders[0],))

		sleepTime = random.randint(1, 15)
		self.log.info("Thread is sleeping %s seconds" % sleepTime)
		self.threadSleep(sleepTime, 15, breakCheck=self.nameQueue.empty)

		while not self.config.stopChildThreads:
			try:
				name = self.nameQueue.get_nowait()

				self.config.progBars.addSubProgressBar()

				self.log.info("Getting %s's Gallery" % name)

				faillinks, successlinks = self.getFA(name, getHandle)

				if faillinks:
					self.failedQueue.put([name, faillinks])

				self.log.info("Done")

				self.config.progBars.removeSubProgressBar()

				self.config.progBars.incMainBar()

				if successlinks != []:										# don't sleep if we didn't download anything
					maxSleepTime = 15 * 60
					minSleepTime = 60
					sleepTime = random.randint(minSleepTime, maxSleepTime)
					self.threadSleep(sleepTime, maxSleepTime, breakCheck=self.nameQueue.empty)



			except Queue.Empty:
				self.log.info("Thread \"", threading.currentThread().name, "\" queue is empty. Exiting")
				break

		self.log.info("Thread ", threading.currentThread(), " has finished.")


	def main(self, config, getHandles):											 # Main Function Call - Uses other functions

		self.log = logmanager.ThreadLog(self.retrievalHandles)

		self.log.info("Starting")


		self.config = config

		self.DLFolder = os.path.join(self.config.downloadDir, self.DLFolder)

		if not os.access(self.DLFolder, os.W_OK):
			try:
				os.mkdir(self.DLFolder)
			except:
				self.log.error("Need write Permissions")
				return




		print "Starting"


		self.config.progBars.setupMainBar(len(self.config.aList))

		self.nameQueue = Queue.Queue()
		self.failedQueue = Queue.Queue()

		for name in self.config.aList:
			self.nameQueue.put(name)
		#print self.nameQueue

		threadNumber = 0
		threads = []

		print "Starting Threads"
		self.config.stopChildThreads = False

		for getHandle in getHandles:
			temp = threading.Thread(target=self.childThread, name = "Thread %d" % threadNumber,  args=(getHandle, ))
			threadNumber += 1
			temp.daemon = True
			temp.start()
			threads.append(temp)

		import time
		alive = True

		while alive:
			alive = False
			for childThread in threads:
				alive = alive or childThread.isAlive()		# Do not exit untill all child threads have returned


			time.sleep(0.1)

		self.log.info("All threads have exited")
		print "All threads have exited"

		totalfaillinks = []
		while not self.failedQueue.empty():
			totalfaillinks.append(self.failedQueue.get())


		faillogptr = open("Failed Pages.txt", "a")

		self.log.info("")
		self.log.info("- - - - - - - - - - -Overall Failed Pages: %s - - - - - - - - - - -" % (len(totalfaillinks)))
		self.log.info("")

		faillogptr.write("\n\n\n- - - - - - - - - - - Successful Pages: %s at %s - - - - - - - - - - -\n" % (len(totalfaillinks), time.ctime(time.time())))

		for link in totalfaillinks:
			if link:
				faillogptr.write("\n		%s" % link)
		for link in totalfaillinks:
			if link:
				self.log.info(link)

		faillogptr.close()
		self.log.info("")
		self.log.info("Done!")
		self.log.info("")


		logmanager.clearThreadLog(self.log)

		print "detached"

